# 单线程代理爬虫（非正式版）


# 猴子补丁
import gevent
from gevent import monkey
from gevent.pool import Pool
monkey.patch_all()
# 导入协程池


import settings
import importlib
from validator.httpbin_validator import check_proxy
from core.mongo_pool import MongoPool
from utils.log import logger


class RunSpider(object):
    def __init__(self):
        self.corotine_pool = Pool()
        self.proxy_pool = MongoPool()

    def get_spider_from_settings(self):
        """根据配置信息, 自动导入爬虫"""
        instances = []
        for full_class_name in settings.PROXIES_SPIDERS:
            module_name, cls_name = full_class_name.rsplit('.', maxsplit=1)
            module = importlib.import_module(module_name)
            # 根据类名, 从模块中, 获取爬虫类
            cls = getattr(module, cls_name)
            # 创建爬虫对象, 添加到列表中
            yield cls()

    def run(self):

        spiders = self.get_spider_from_settings()
        for spider in spiders:
            self.__execute_one_spider_task(spider)
            self.corotine_pool.apply_async(self.__execute_one_spider_task, args=(spider,))
        self.corotine_pool.join()

    def __execute_one_spider_task(self, spider):
        """用于处理一个爬虫任务"""
        try:
            for proxy in spider.get_proxies():
                proxy = check_proxy(proxy)
                if proxy.speed != -1:
                    self.proxy_pool.insert_one(proxy)
        except Exception as ex:
            logger.exception(ex)


if __name__ == '__main__':
    rs = RunSpider()
    rs.run()
